# -*- coding: utf-8 -*-

import scrapy
from proxy_ip.items import Xi_Ci_Item

from urllib import parse
import time
from scrapy.http import Request
from scrapy.loader import ItemLoader

class XiciSpider(scrapy.Spider):
    name = 'xici'
    allowed_domains = ['www.xicidaili.com']
    start_urls = ['http://www.xicidaili.com/wn/']
    # allowed_domains = ['shudong.phpcool.cn']
    # start_urls = ['http://shudong.phpcool.cn/show_ip']
    handle_httpstatus_list = [404, 501, 502, 503, 403]

    def parse(self, response):
        list = response.css("#ip_list tr")
        if (list):
            for li in list[1:]:
                all_info = li.css("td::text").extract()
                item = Xi_Ci_Item()
                item['ip'] = all_info[0]
                item['port'] = all_info[1]
                item['proxy_type'] = all_info[5]
                item['check_time'] = all_info[11]
                yield item
        pass
